﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using PasteTimer.HttpApi.Host.Service;
using PasteTimer.slavemodels;
using PasteTimer.taskmodels;
using Z.EntityFramework.Plus;

namespace PasteTimer
{
    /// <summary>
    /// 集群模式
    /// </summary>
    public class SlaveHostedService : IHostedService
    {

        /// <summary>
        /// 
        /// </summary>
        private IServiceProvider _serviceProvider;
        /// <summary>
        /// 
        /// </summary>
        private Random _random;
        private SlaveHelper _slaveHelper;
        private readonly SystemStartHandler _systemStartHandler;
        private ILogger<SlaveHostedService> _logger;
        /// <summary>
        /// 
        /// </summary>
        private readonly IHttpClientFactory _httpClientFactory;
        /// <summary>
        /// 队列状态等
        /// </summary>
        private SemaphoreSlim _semplorestate;
        /// <summary>
        /// 
        /// </summary>
        private Volo.Abp.ObjectMapping.IObjectMapper _objectMapper;
        /// <summary>
        /// 
        /// </summary>
        private ChannelHelper _channelHelper;
        /// <summary>
        /// 健康检查的并发量
        /// </summary>
        private readonly SemaphoreSlim _semaphorehealth;
        /// <summary>
        /// 同时对外工作的并发量
        /// </summary>
        private readonly SemaphoreSlim _semaphorework;
        private readonly SemaphoreSlim _semaphoreretry;
        private IAppCache _cache;
        private TaskConfig _config;
        private readonly ModelHelper _modelHelper;

        /// <summary>
        /// 
        /// </summary>
        /// <param name="serviceProvider"></param>
        /// <param name="httpClientFactory"></param>
        /// <param name="logger"></param>
        /// <param name="objectMapper"></param>
        /// <param name="channelHelper"></param>
        /// <param name="config"></param>
        /// <param name="modelHelper"></param>
        /// <param name="systemStartHandler"></param>
        /// <param name="slaveHelper"></param>
        /// <param name="appCacheHelper"></param>
        public SlaveHostedService(IServiceProvider serviceProvider,
            IHttpClientFactory httpClientFactory,
            ILogger<SlaveHostedService> logger,
            Volo.Abp.ObjectMapping.IObjectMapper objectMapper,
            ChannelHelper channelHelper,
            IOptions<TaskConfig> config,
            ModelHelper modelHelper,
            SystemStartHandler systemStartHandler,
            SlaveHelper slaveHelper,
            IAppCache appCacheHelper)
        {
            _serviceProvider = serviceProvider;
            _httpClientFactory = httpClientFactory;
            _logger = logger;
            _objectMapper = objectMapper;
            _channelHelper = channelHelper;

            _random = new Random();
            _config = config.Value;
            _cache = appCacheHelper;
            _semplorestate = new SemaphoreSlim(1);
            _semaphorehealth = new SemaphoreSlim(20);
            dicretrylist = new Dictionary<long, List<RetryModelItem>>();
            _semaphorework = new SemaphoreSlim(20);
            _systemStartHandler = systemStartHandler;
            _semaphoreretry = new SemaphoreSlim(10);
            _slaveHelper = slaveHelper;
            _modelHelper = modelHelper;
        }

        private System.Timers.Timer _timertick;
        private List<TaskInfoDto> _tasklist;
        private List<NodeInfoDto> _nodelist;

        /// <summary>
        /// slave健康检查计时器
        /// </summary>
        private System.Timers.Timer _timerhealth;

        /// <summary>
        /// 
        /// </summary>
        /// <param name="cancellationToken"></param>
        /// <returns></returns>
        public Task StartAsync(CancellationToken cancellationToken)
        {
            Console.WriteLine($"{DateTime.Now} SlaveHostedService Start! MachineName:{Environment.MachineName}");
            _tasklist = new List<TaskInfoDto>();
            _nodelist = new List<NodeInfoDto>();
            SystemInstance();
            _timertick = new System.Timers.Timer();
            _timertick.Interval = 1000;
            _timertick.Elapsed += _timertick_Elapsed;
            _timertick.AutoReset = true;
            if (_config.SingleModel)
            {
                _timertick.Start();
                InstanceInit(true);
            }
            else
            {
                //集群心跳包监测
                _timerhealth = new System.Timers.Timer();
                _timerhealth.Interval = 30000;
                _timerhealth.Elapsed += _timerhealth_Elapsed;
                _timerhealth.AutoReset = true;
                _timerhealth.Start();
                FindMySlave();
            }
            ReadChannel();
            return Task.CompletedTask;
        }

        private async void SystemInstance()
        {
            await _systemStartHandler.SystemStart();
        }

        /// <summary>
        /// 集群心跳包监测
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        private async void _timerhealth_Elapsed(object sender, System.Timers.ElapsedEventArgs e)
        {
            if (_slaveHelper.IsFind)
            {
                if (!_slaveHelper.IsMaster)
                {
                    //向master汇报心跳包
                    if (_slaveHelper.Master != null)
                    {
                        //有管理者
                        var result = await _slaveHelper.Health();
                        if (result.code == 500)
                        {
                            await StartVote();
                        }
                    }
                    else
                    {
                        //没有管理者 去vote选举
                        await StartVote();
                    }
                }
                else
                {
                    //检查哪一个节点超时了

                }
            }
            else
            {
                //还没有发现自己？
                FindMySlave();
            }

        }

        private long _LastSecond = -1;

        /// <summary>
        /// 初始化数据 程序缓存任务和节点数据
        /// </summary>
        private async void InstanceInit(bool init = false)
        {
            if (init)
            {
                await Task.Delay(5000);
            }
            using var _scope = _serviceProvider.CreateScope();
            using var _dbContext = _scope.ServiceProvider.GetRequiredService<IPasteTimerDbContext>();
            var nowdate = DateTime.Now;
            var _task = await _dbContext.TaskInfo.Where(x => x.EndDate >= nowdate && x.IsEnable).AsNoTracking().ToListAsync();
            if (_task != null && _task.Count > 0)
            {
                _objectMapper.Map(_task, _tasklist);
            }
            var _node = await _dbContext.NodeInfo.Where(x => x.IsEnable).AsNoTracking().ToListAsync();
            if (_node != null && _node.Count > 0)
            {
                _objectMapper.Map(_node, _nodelist);
            }
            _dbContext.Dispose();
            _scope.Dispose();
            Console.WriteLine($"read tasklist.count:{_tasklist.Count} nodelist.count:{_nodelist.Count}");
        }


        /// <summary>
        /// 开始选举
        /// </summary>
        /// <returns></returns>
        private async Task<int> StartVote()
        {
            using var _scope = _serviceProvider.CreateScope();
            using var _dbContext = _scope.ServiceProvider.GetRequiredService<IPasteTimerDbContext>();
            var nodes = await _dbContext.NodeInfo.Where(x => x.Id != _slaveHelper.MySlaveId && x.IsEnable && x.Group == "slave").AsNoTracking().ToListAsync();
            if (nodes != null && nodes.Count > 0)
            {
                var slavelistdto = _objectMapper.Map<List<NodeInfo>, List<SlaveInfoDto>>(nodes);
                await _slaveHelper.StartVote(slavelistdto);
            }
            else
            {
                //选举自己为管理者，因为只有自己一个
                _slaveHelper.SetMaster(_slaveHelper.MyMaster);
            }
            if (_slaveHelper.IsMaster)
            {
                _channelHelper.WriteSlaveAction(new SlaveEventModel() { Event = "startmaster", Object = "" });
            }
            _dbContext.Dispose();
            _scope.Dispose();
            return 1;
        }

        /// <summary>
        /// 找到自己
        /// </summary>
        /// <returns></returns>
        private async void FindMySlave()
        {
            using var _scope = _serviceProvider.CreateScope();
            using var _dbContext = _scope.ServiceProvider.GetRequiredService<IPasteTimerDbContext>();
            var nodes = await _dbContext.NodeInfo.Where(x => x.Id != _slaveHelper.MySlaveId && x.IsEnable && x.Group == "slave").AsNoTracking().ToListAsync();
            if (nodes != null && nodes.Count > 0)
            {
                var slavelistdto = _objectMapper.Map<List<NodeInfo>, List<SlaveInfoDto>>(nodes);
                await _slaveHelper.FindMe(slavelistdto);
            }
            if (_slaveHelper.MyMaster != null)
            {
                //var coll = _serviceProvider.GetRequiredService<IServiceCollection>();
                //准备加入到集群中，启动自己节点的身份
                //加入集群
                await StartVote();
            }
            _dbContext.Dispose();
            _scope.Dispose();
        }

        /// <summary>
        /// 
        /// </summary>
        private async void ReadChannel()
        {
            try
            {
                var info = await _channelHelper.Slave.Reader.ReadAsync();
                if (info != null && info != default)
                {
                    switch (info.Event)
                    {
                        case "status":
                            {
                                if (_config.SingleModel)
                                {
                                    var one = Newtonsoft.Json.JsonConvert.DeserializeObject<StatusModel>(info.Object);
                                    await UpdateStatus(one);
                                }
                                else
                                {
                                    if (_slaveHelper.IsMaster)
                                    {
                                        var one = Newtonsoft.Json.JsonConvert.DeserializeObject<StatusModel>(info.Object);
                                        await UpdateStatus(one);
                                    }
                                    else
                                    {
                                        await _slaveHelper.PostAction(info);
                                    }
                                }
                            }
                            break;
                        case "startmaster":
                            {
                                _LastSecond = DateTimeOffset.Now.ToUnixTimeSeconds();
                                //自己是管理员
                                InstanceInit();
                                _timertick.Start();
                            }
                            break;
                        case "stopmaster":
                            {
                                //自己不是管理员了
                                _timertick.Stop();
                            }
                            break;
                        case "votemaster":
                            {
                                //被通知竞选管理者
                                await StartVote();
                            }
                            break;
                        case "testtask":
                            {
                                //测试任务
                                var _taskdto = Newtonsoft.Json.JsonConvert.DeserializeObject<TaskInfoDto>(info.Object);
                                if (_taskdto != null && _taskdto != default)
                                {
                                   await SelectNodeToTask(_taskdto);
                                }
                            }
                            break;
                        default:
                            //其他事件的动作
                            break;
                    }
                }
            }
            catch (Exception exl)
            {
                _logger.LogException(exl);
                await Task.Delay(1000);
            }
            finally
            {
                ReadChannel();
            }
        }

        /// <summary>
        /// 处理任务和节点的状态变更
        /// </summary>
        /// <param name="info"></param>
        /// <returns></returns>
        private async Task<int> UpdateStatus(StatusModel info)
        {
            try
            {
                _logger.LogInformation($"UpdateStatus at Action:{info.Action}");
                _logger.LogInformation(info.body);
                await _semplorestate.WaitAsync();
                if (info.ModelType == 1)
                {
                    //task
                    switch (info.Action)
                    {
                        case 1:
                            {
                                var task = Newtonsoft.Json.JsonConvert.DeserializeObject<TaskInfoDto>(info.body); //info.body as TaskInfoDto;
                                if (task != null)
                                {
                                    var find = _tasklist.Where(x => x.Id == task.Id).FirstOrDefault();
                                    if (find != null && find != default)
                                    {
                                        _tasklist.Remove(find);
                                    }
                                    _tasklist.Add(task);
                                }
                            }
                            break;
                        case 2:
                            {
                                var task = Newtonsoft.Json.JsonConvert.DeserializeObject<TaskInfoDto>(info.body);// info.body as TaskInfoDto;
                                if (task != null)
                                {
                                    var find = _tasklist.Where(x => x.Id == task.Id).FirstOrDefault();
                                    if (find != null && find != default)
                                    {
                                        _tasklist.Remove(find);
                                    }
                                    _tasklist.Add(task);
                                }
                            }
                            break;
                        case 3:
                            {
                                //var id = (int)info.body;// as Int32;
                                int.TryParse(info.body, out int id);
                                if (id != 0)
                                {
                                    var find = _tasklist.Where(x => x.Id == id).FirstOrDefault();
                                    if (find != null && find != default)
                                    {
                                        _tasklist.Remove(find);
                                    }
                                }
                            }
                            break;
                    }
                }
                else if (info.ModelType == 2)
                {
                    //node
                    switch (info.Action)
                    {
                        case 1:
                            {
                                var task = Newtonsoft.Json.JsonConvert.DeserializeObject<NodeInfoDto>(info.body);// info.body as NodeInfoDto;
                                if (task != null)
                                {
                                    var find = _nodelist.Where(x => x.Id == task.Id).FirstOrDefault();
                                    if (find != null && find != default)
                                    {
                                        find.Status = task.Status;
                                    }
                                    else
                                    {
                                        _nodelist.Add(task);
                                    }
                                }
                            }
                            break;
                        case 2:
                            {
                                var task = Newtonsoft.Json.JsonConvert.DeserializeObject<NodeInfoDto>(info.body);// info.body as NodeInfoDto;
                                if (task != null)
                                {
                                    var find = _nodelist.Where(x => x.Id == task.Id).FirstOrDefault();
                                    if (find != null && find != default)
                                    {
                                        _nodelist.Remove(find);
                                    }
                                    _nodelist.Add(task);
                                }
                            }
                            break;
                        case 3:
                            {
                                int.TryParse(info.body, out int id);
                                //var id = (int)info.body;// as Int32;
                                if (id != 0)
                                {
                                    var find = _nodelist.Where(x => x.Id == id).FirstOrDefault();
                                    if (find != null && find != default)
                                    {
                                        _nodelist.Remove(find);
                                    }
                                }
                            }
                            break;
                    }
                }
            }
            finally
            {
                _semplorestate.Release();
            }
            return 1;
        }

        /// <summary>
        /// 事件计时器
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        private void _timertick_Elapsed(object sender, System.Timers.ElapsedEventArgs e)
        {
            try
            {
                //这里有问题！时间差，可能跳过...

                //await _semplorestate.WaitAsync();
                var nowdate = DateTimeOffset.Now;
                var nowlong = nowdate.ToUnixTimeSeconds();
                if (_LastSecond == -1) { _LastSecond = nowlong - 1; }
                if (_LastSecond + 1 == nowlong)
                {
                    CheckTickTask(nowlong, nowdate.DateTime);
                }
                else
                {
                    if (_LastSecond < nowlong)
                    {
                        _logger.LogWarning($"--- --- {DateTime.Now} find jump tick from:{_LastSecond} to:{nowlong} --- ---");
                        for (var k = _LastSecond + 1; k <= nowlong; k++)
                        {
                            CheckTickTask(k, DateTimeOffset.FromUnixTimeSeconds(k).DateTime);
                        }
                    }
                }
                _LastSecond = nowlong;
                ActionRetryWork(nowlong);
            }
            catch (Exception exl)
            {
                _logger.LogException(exl);
            }
            finally
            {
                //_semplorestate.Release();
            }
        }

        /// <summary>
        /// 执行重试
        /// </summary>
        /// <param name="date"></param>
        private async void ActionRetryWork(long date)
        {
            try
            {
                await _semaphoreretry.WaitAsync();
                if (dicretrylist.ContainsKey(date))
                {
                    var values = dicretrylist[date];
                    foreach (var item in values)
                    {
                        var task = item.task;
                        task.PlanTime = date;
                        await SelectNodeToTask(task, item.timeindex);
                    }
                }
            }
            catch (Exception exl)
            {
                _logger.LogException(exl);
            }
            finally
            {
                _semaphoreretry.Release();
            }
        }

        /// <summary>
        /// 计算任务是否被时间命中
        /// </summary>
        /// <param name="time"></param>
        /// <param name="dt"></param>
        private async void CheckTickTask(long time, DateTime dt)
        {
            try
            {
                //计时模式
                var hittask = _tasklist.Where(x => x.TickSecond != 0 && x.IsEnable == true && x.StartDate <= dt && x.EndDate >= dt && (time % x.TickSecond == 0)).ToList();
                if (hittask != null && hittask.Count > 0)
                {
                    foreach (var item in hittask)
                    {
                        item.TaskDate = dt;
                        item.TaskTime = time;
                        await SelectNodeToTask(item);
                    }
                }

                //正则模式 时间点执行也是计算在这里的  比如2023-12-11 11:11:11执行这个任务
                var timelist = _tasklist.Where(x => x.TickSecond == 0 && x.IsEnable == true && x.StartDate <= dt && dt <= x.EndDate).ToList();
                if (timelist != null && timelist.Count > 0)
                {
                    //比较时间，看看是否命中
                    var timestr = $"{dt.DayOfWeek.ToString().Substring(0, 3)} {dt.ToString("yyyy-MM-dd HH:mm:ss")}";
                    foreach (var item in timelist)
                    {
                        if (new System.Text.RegularExpressions.Regex(item.TickRegex).Match(timestr).Success)
                        {
                            item.TaskDate = dt;
                            item.TaskTime = time;
                            await SelectNodeToTask(item);
                        }
                    }
                }
            }
            catch (Exception exl)
            {
                _logger.LogException(exl);
            }
        }

        /// <summary>
        /// 选择一个节点执行任务
        /// </summary>
        /// <param name="task"></param>
        /// <param name="tryindex"></param>
        private async Task SelectNodeToTask(TaskInfoDto task, int tryindex = 0)
        {
            _modelHelper.CollectData(task.Id, CollectType.run);
            var _find_node = false;
            if (task != null && task != default)
            {
                if (_nodelist != null && _nodelist.Count > 0)
                {
                    task.TaskLogGuid = Guid.NewGuid().ToString();
                    var log = new TaskLog();
                    log.TaskId = task.Id;
                    log.CreateDate = DateTime.Now;
                    log.Status = TaskStatus.waitrun;
                    if (task.Groups == "*")
                    {
                        var tempnode = _nodelist.Where(x => x.Status == NodeStatus.running && x.Group != "slave").ToList();
                        if (tempnode != null && tempnode.Count > 0)
                        {
                            var hitnode = tempnode[_random.Next(0, tempnode.Count)];
                            if (hitnode != null && hitnode != default)
                            {
                                _find_node = true;
                                log.NodeId = hitnode.Id;
                                await _senddowork(task, hitnode, log, tryindex);
                            }
                        }
                    }
                    else
                    {
                        if (!String.IsNullOrEmpty(task.Groups))
                        {
                            var groups = task.Groups.Split(',');
                            var tempnode = _nodelist.Where(x => groups.Contains(x.Group) && x.Status == NodeStatus.running).ToList();
                            if (tempnode != null && tempnode.Count() > 0)
                            {
                                var hitnode = tempnode[_random.Next(0, tempnode.Count())];
                                if (hitnode != null && hitnode != default)
                                {
                                    _find_node = true;
                                    log.NodeId = hitnode.Id;
                                    await _senddowork(task, hitnode, log, tryindex);
                                }
                            }
                        }
                    }
                }
                if (!_find_node)
                {
                    _channelHelper.WriteNotice(new noticemodels.NoticeLogAddDto() { Body = $"没有找到可以为任务 id:{task.Id} name:{task.Assembly} 工作的节点，任务推送失败！", Code = "nodenotfound", ObjId = task.Id });
                    _logger.LogError($"not found node to work task:{Newtonsoft.Json.JsonConvert.SerializeObject(task)}");
                    _modelHelper.CollectData(task.Id, CollectType.fail);
                }
            }
        }

        /// <summary>
        /// 向节点发送一个任务
        /// </summary>
        /// <param name="task"></param>
        /// <param name="node"></param>
        /// <param name="log"></param>
        /// <param name="tryindex">当前第几次重试</param>
        private async Task _senddowork(TaskInfoDto task, NodeInfoDto node, TaskLog log, int tryindex = 0)
        {
            _logger.LogInformation($"begin send work:{task.Id} request to:{node.Id} url:{node.Url} zindex:{tryindex}");
            try
            {
                await _semaphorework.WaitAsync();
                var stop = new System.Diagnostics.Stopwatch();
                stop.Start();
                log.Status = TaskStatus.running;
                using var _scope = _serviceProvider.CreateScope();
                using var _dbContext = _scope.ServiceProvider.GetRequiredService<IPasteTimerDbContext>();
                _dbContext.Add(log);
                _dbContext.SaveChanges();
                task.TaskLogId = (int)log.Id;
                task.NodeId = node.Id;
                var post = await SendToNode(task, node);
                if (post)
                {
                    log.Status = TaskStatus.waitrun;
                }
                else
                {
                    _modelHelper.CollectData(task.Id, CollectType.fail);
                    if (task.RetryCount > 0 && task.RetryCount > tryindex)
                    {

                        //写入回新的队列中
                        AppendToRetryArray(task, tryindex);
                    }
                    log.Status = TaskStatus.failed;
                }
                stop.Stop();
                _dbContext.TaskLog.Where(x => x.Id == log.Id && x.Status == TaskStatus.running).Update(x => new TaskLog() { Status = log.Status, Durtion = (int)stop.Elapsed.TotalSeconds });
                if (task.TaskType == TaskType.TimeSpan)
                {
                    await _dbContext.TaskInfo.Where(x => x.Id == task.Id).UpdateAsync(x => new TaskInfo { IsEnable = false });

                    _tasklist.Remove(task);
                }
                _dbContext.Dispose();
                _scope.Dispose();
            }
            catch (Exception exl)
            {
                _modelHelper.CollectData(task.Id, CollectType.fail);
                _logger.LogException(exl);
            }
            finally
            {
                _semaphorework.Release();
            }
        }

        /// <summary>
        /// 失败后加入到重试队列中
        /// </summary>
        /// <param name="task"></param>
        /// <param name="tryindex">当前尝试的次数</param>
        private void AppendToRetryArray(TaskInfoDto task, int tryindex)
        {
            try
            {
                var nowindex = tryindex + 1;
                var nexttime = (task.PlanTime == 0 ? task.TaskTime : task.PlanTime) + ReadNextWorkSecond(nowindex);
                _semaphoreretry.Wait();
                if (dicretrylist.ContainsKey(nexttime))
                {
                    var val = dicretrylist[nexttime];
                    val.Add(new RetryModelItem()
                    {
                        task = task,
                        timeindex = nowindex,
                        timepoint = nexttime
                    });
                }
                else
                {
                    dicretrylist.Add(nexttime, new List<RetryModelItem>
                    {
                        new RetryModelItem()
                        {
                     task=task,
                      timeindex=nowindex,
                        timepoint=nexttime
                    } });
                }
            }
            catch (Exception exl)
            {
                _logger.LogException(exl);
            }
            finally
            {
                _semaphoreretry.Release();
            }
        }

        /// <summary>
        /// 单次发送任务给远程地址节点
        /// </summary>
        /// <param name="task"></param>
        /// <param name="node"></param>
        /// <returns></returns>
        private async Task<bool> SendToNode(TaskInfoDto task, NodeInfoDto node)
        {
            try
            {
                var _postbody = (Newtonsoft.Json.JsonConvert.SerializeObject(task));
                using HttpContent httpcontent = new StringContent(_postbody);
                var timestmp = DateTimeOffset.Now.ToUnixTimeSeconds();
                var headtoken = ($"{timestmp}_{node.Id}_{node.JoinToken}").ToMd5Lower();
                var xtoken = $"{timestmp}_{node.Id}_{headtoken}";//服务端需要根据游戏ID获取 密钥，否则无法验证xtoken是否合法
                httpcontent.Headers.Add("xtoken", xtoken);
                httpcontent.Headers.ContentType = System.Net.Http.Headers.MediaTypeHeaderValue.Parse("application/json;charset=utf-8");
                var client = _httpClientFactory.CreateClient();
                client.Timeout = TimeSpan.FromSeconds(10);
                var url = $"{node.Url}{task.HttpPath}";
                switch (task.RunWay)
                {
                    case RunWay.Assembly:
                    case RunWay.RemoteAssembly:
                        {
                            url = $"{node.Url}api/taskclient/dowork";
                        }
                        break;
                }
                using var response = await client.PostAsync(url, httpcontent);
                if (response.IsSuccessStatusCode)
                {
                    return true;
                }
                else
                {
                    _channelHelper.WriteNotice(new noticemodels.NoticeLogAddDto() { Body = $"任务:{task.Id} 发送给:{node.Url} 失败！ 异常：{response.StatusCode} {await response.Content.ReadAsStringAsync()}", Code = "workfail", ObjId = node.Id });
                    return false;
                }
            }
            catch (Exception exl)
            {
                _logger.LogError($"{DateTime.Now} throw exception at line 603 node.id:{node.Id} node.url:{node.Url}");
                _logger.LogException(exl);
                _channelHelper.WriteNotice(new noticemodels.NoticeLogAddDto() { Body = $"任务:{task.Id} 发送给:{node.Url} 失败！ 异常：{exl.Message}", Code = "sendexception", ObjId = node.Id });
                return false;
            }
        }

        /// <summary>
        ///健康检查 
        ///如何自动恢复？
        ///使用延迟队列，确定下一次测试的时间，时间为周期的1/2 如果失败了，则加一倍时间，最高加8小时！ 最低频率10秒钟
        /// </summary>
        /// <param name="node"></param>
        /// <returns>是否需要保存状态</returns>
        private async void TickLink(NodeInfoDto node)
        {
            var boolchange = false;
            try
            {
                await _semaphorehealth.WaitAsync();
                var timestmp = DateTimeOffset.Now.ToUnixTimeSeconds();
                var headtoken = ($"{timestmp}_{node.Id}_{node.JoinToken}");
                var xtoken = $"{timestmp}_{node.Id}_{headtoken}";//服务端需要根据游戏ID获取 密钥，否则无法验证xtoken是否合法
                var client = _httpClientFactory.CreateClient();
                var url = $"{node.Url}api/taskclient/health";
                using HttpRequestMessage hrm = new HttpRequestMessage(HttpMethod.Get, url);
                hrm.Headers.Add("xtoken", xtoken);
                client.Timeout = TimeSpan.FromSeconds(20);
                using var response = await client.SendAsync(hrm);
                if (response.IsSuccessStatusCode)
                {
                    node.LastLink = DateTimeOffset.Now.ToUnixTimeSeconds();
                    if (node.Status != NodeStatus.running)
                    {
                        node.Status = NodeStatus.running;
                        //应该要同步给服务端
                        boolchange = true;
                    }
                    node.TryTime = 0;
                }
                else
                {
                    node.TryTime++;
                    //Console.WriteLine($"{DateTime.Now} tick node id:{node.Id} health falied at code:{response.StatusCode}");
                }
            }
            catch (Exception exl)
            {
                _logger.LogException(exl);
                node.TryTime++;
                node.Status = NodeStatus.stop;
                //应该要同步给服务端
                boolchange = true;
                _channelHelper.WriteNotice(new noticemodels.NoticeLogAddDto() { Body = $"对节点{node.Url}的健康检查失败！异常：{exl.Message}", Code = "healthexception", ObjId = (node != null ? node.Id : 0) });
            }
            finally
            {
                _semaphorehealth.Release();

                if (boolchange || (node.Status == NodeStatus.running && node.TryTime >= _config.TryConnectTime))
                {
                    if (node.TryTime >= _config.TryConnectTime)
                    {
                        node.Status = NodeStatus.error;

                        _logger.LogError($"{DateTime.Now} node.id:{node.Id} node.url:{node.Url} connect error out of time {_config.TryConnectTime}");
                    }
                    using var _scope = _serviceProvider.CreateScope();
                    using var _dbContext = _scope.ServiceProvider.GetRequiredService<IPasteTimerDbContext>();
                    await _dbContext.NodeInfo.Where(x => x.Id == node.Id).UpdateAsync(x => new NodeInfo() { Status = node.Status });
                    _dbContext.Dispose();
                    _scope.Dispose();
                }
            }
        }

        ////using System.Runtime.Loader
        ///// <summary>
        ///// 从文件中加载程序集
        ///// </summary>
        ///// <param name="className"></param>
        ///// <param name="pluginLocation"></param>
        ///// <returns></returns>
        //private IBaseClass ReadBaseClassFromFile(string className, string pluginLocation)
        //{
        //    //$"{Directory.GetCurrentDirectory()}{path}".Replace('\\', Path.DirectorySeparatorChar);
        //    //string pluginLocation = GetTaskAssemblyPath(sid, assemblyName);
        //    var assembly = context.LoadFromAssemblyName(new AssemblyName(Path.GetFileNameWithoutExtension(pluginLocation)));
        //    Type type = assembly.GetType(className, true, true);
        //    return Activator.CreateInstance(type) as IBaseClass;
        //}
        //var score = _serviceProvider.CreateScope();
        //var tts = score.ServiceProvider.GetServices<IBaseClass>();
        //foreach (var item in tts)
        //{
        //    var t = item.GetType();
        //    Console.WriteLine($"Name:{t.Name} Namespace:{t.Namespace} FullName:{t.FullName}");
        //    item.Read();
        //}
        //        protected void invokeTheMethod(object theObject, string strMethod, object[] objParms)
        //        {
        //            Type objType = theObject.GetType();  //--得到对象的类型 cyj 2011-3-28
        //            MethodInfo method;  //--声明一个方法 cyj 2011-3-28
        //            try
        //            {
        //                method = objType.GetMethod(strMethod);  //--根据方法名称得到方法 cyj 2011-3-28
        //                method.Invoke(theObject, objParms);     //--连接类型与方法 cyj 2011-3-28
        //            }
        //            catch (Exception)
        //            {
        //                throw (new Exception("刷新分页控件出错"));
        //                // Response.Write(ex.ToString());
        //            }
        //        }
        //————————————————
        //版权声明：本文为CSDN博主「longxin5」的原创文章，遵循CC 4.0 BY-SA版权协议，转载请附上原文出处链接及本声明。
        //原文链接：https://blog.csdn.net/longxin5/article/details/83880218

        /// <summary>
        /// 离线了，如果自己是管理者，则随机获取下一个节点进行告知
        /// </summary>
        /// <param name="cancellationToken"></param>
        /// <returns></returns>
        public async Task StopAsync(CancellationToken cancellationToken)
        {
            try
            {
                //随机告知一个slave,我即将下线
                if (_slaveHelper.MySlaveId != 0)
                {
                    //using var _scope = _serviceProvider.CreateScope();
                    using var _dbContext = _serviceProvider.GetRequiredService<IPasteTimerDbContext>();
                    await _dbContext.NodeInfo.Where(x => x.Id == _slaveHelper.MySlaveId).UpdateAsync(x => new NodeInfo() { Status = NodeStatus.stop });
                    if (_slaveHelper.IsMaster)
                    {
                        //随机告知下一个节点去竞选管理者
                        var nodes = _dbContext.NodeInfo.Where(x => x.Group == "slave" && x.Status == NodeStatus.running).AsNoTracking().ToList();
                        if (nodes != null && nodes.Count > 0)
                        {
                            foreach (var item in nodes)
                            {
                                var result = await _slaveHelper.PostOffline(new SlaveMaster() { id = item.Id, time = 0, token = item.Token, url = item.Url });
                                if (result.code == 200)
                                {
                                    break;
                                }
                            }
                        }
                    }
                    else
                    {
                        //告知管理者，我这个slave将要下线了，master以便于处理工作
                        //TaskRemoteHostedService会执行这个命令
                    }
                    _dbContext.Dispose();
                }
            }
            finally
            {
            }
        }

        /// <summary>
        /// 任务重试队列
        /// </summary>
        private Dictionary<long, List<RetryModelItem>> dicretrylist;

        /// <summary>
        /// 计算下一次的重试时间点
        /// </summary>
        /// <param name="time"></param>
        /// <returns></returns>
        private int ReadNextWorkSecond(int time)
        {
            switch (time)
            {
                case 1: return 5;
                case 2: return 10;
                case 3: return 20;
                case 4: return 45;
                case 5: return 60;
                default: return 120;
            }
        }
    }
    /// <summary>
    /// 任务重试对象
    /// </summary>
    public class RetryModelItem
    {
        /// <summary>
        /// 任务情况
        /// </summary>
        public TaskInfoDto task { get; set; }

        /// <summary>
        /// 第几次重试
        /// </summary>
        public int timeindex { get; set; } = 1;

        /// <summary>
        /// 执行时间点 秒数
        /// </summary>
        public long timepoint { get; set; }
    }


}
